module lang.d.lexer.dlexer;

import utils.log;

import lang.ilexer;
import lang.d.tokens.tokentype;
import lang.d.tokens.token;
import lang.d.tokens.patterns;
import character;
import traits;
import std.utf;
import std.typetuple;
import std.variant;
import std.math;
//import std.traits;
debug import std.stdio;

// TODO: Support special token sequence
class DLexer: ILexer {
    private static class TextDriver {
        private string myText;
        private char myChar;
        private uint myPosition;

        invariant() {
            assert(myPosition <= myText.length);
        }

        public this(in string text = "") {
            myText = text;
            if (myText.length != 0) {
                myChar = myText[0];
            }
        }

        public void setText(in string text) {
            myText = text;
            myPosition = 0;
            if (myText.length != 0) {
                myChar = myText[0];
            }
        }

        public void nextChar() {
            ++myPosition;
        }

        public char getChar() const {
            return (myPosition < myText.length) ? myText[myPosition]: '\0';
        }

        public void ungetChar() {
            --myPosition;
        }

        public uint getPosition() const {
            return myPosition;
        }
    }

    private static class LexerException: Exception {
        public this(in string msg) {
            super(msg);
        }
    }

    private static T parseNumber(T)(in string value, in ubyte radix = 10) if (isNumeric!T || isImaginary!T) {
        T simpleParse(T)(in string value, in ubyte radix = 10) {
            T result = 0;
            static if (isIntegral!T) {
                foreach (c; value) {
                    if (!Character.isDigit(c, radix)) {
                        throw new LexerException("Non valid integer literal");
                    }
                    if (result < T.max / radix) {
                        result *= radix;
                        result += Character.toDigit(c);
                        continue;
                    }
                    throw new LexerException("Integer overflow");
                }
            } else {
                bool hasPoint = false;
                bool hasExponent = false;
                real base = radix;
                foreach (i, c; value) {
                    if (c == '.') {
                        hasPoint = true;
                        continue;
                    }
                    if ((radix != 16) && (c == 'e' || c == 'E')) {
                        hasExponent = true;
                        continue;
                    }
                    if (c == 'p' || c == 'P') {
                        if (radix != 16) {
                            throw new LexerException("Non valid float literal");
                        }
                        hasExponent = true;
                        continue;
                    }
                    if (!Character.isDigit(c, radix)) {
                        if (!hasExponent || (c != '+' && c != '-')) {
                            throw new LexerException("Non valid float literal");
                        }
                    }

                    if (hasExponent) {
                        byte sign = 1;
                        base = (radix != 16) ? 10 : 2;
                        uint pos = i;
                        if (c == '+') {
                            ++pos;
                        } else if (c == '-') {
                            sign = -1;
                            ++pos;
                        }
                        int p = simpleParse!int(value[pos..$]) * sign;
                        if (base == 10) {
                            if (p > T.max_10_exp || p < T.min_10_exp) {
                                throw new LexerException("Number is not represantable");
                            }
                        } else {
                            if (p > T.max_exp || p < T.min_exp) {
                                throw new LexerException("Number is not represantable");
                            }
                        }
                        result *= pow(base, p);
                        break;
                    }

                    if (hasPoint) {
                        result += Character.toDigit(c) / base;
                        base *= radix;
                    } else {
                        result *= radix;
                        result += Character.toDigit(c);
                    }
                }
            }
            return result;
        }

        static if (isImaginary!T) {
            static if (staticIndexOf!(Unqual!(T), ifloat) >= 0) {
                return simpleParse!float(value, radix) * 1.0i;
            } else static if (staticIndexOf!(Unqual!(T), idouble) >= 0) {
                return simpleParse!double(value, radix) * 1.0i;
            } else {
                return simpleParse!real(value, radix) * 1.0i;
            }
        } else {
            return simpleParse!T(value, radix);
        }
    }

    private TextDriver myDriver;
    private Token myCurrentToken;
    private Variant myTokenValue;
    private uint myLineNumber;

    public this() {
        myDriver = new TextDriver();
    }

    public this(in string text) {
        myDriver = new TextDriver(text);
    }

    public void analize(in string text) {
        myDriver.setText(text);
    }

    override public Token getToken() const
    in {
        assert(myCurrentToken !is null);
    }
    body {
        return myCurrentToken;
    }

    override public Token nextToken()
    in {
        assert(myDriver !is null);
    }
    body {
        myTokenValue = "";
        uint start = myDriver.getPosition();
        TokenType tok = scan();
        myCurrentToken = new Token(tok, start, myDriver.getPosition(), myTokenValue);
        return myCurrentToken;
    }

    public TokenType getTokenType()
    in {
        assert(myCurrentToken !is null);
    }
    body {
        while (myCurrentToken.getTokenType() == TokenType.WHITESPACE) {
            nextToken();
        }
        return myCurrentToken.getTokenType();
    }

    private TokenType scan() {
        char c = myDriver.getChar();

        if (Character.isSpace(c)) {
            return scanWhitespaces();
        }

        switch (c) {
        case '\"':
            return scanDoubleQuotedString();

        case '`':
            return scanWysiwygString();

        case '\'':
            return scanSingleCharacter();

        case 'r':
            myDriver.nextChar();
            c = myDriver.getChar();
            if (c == '\"') {
                return scanWysiwygString();
            }
            myDriver.ungetChar();
            return scanWord();

        case 'x':
            myDriver.nextChar();
            c = myDriver.getChar();
            if (c == '\"') {
                return scanHexString();
            }
            myDriver.ungetChar();
            return scanWord();

        case 'q':
            myDriver.nextChar();
            c = myDriver.getChar();
            if (c == '\"') {
                return scanDelemitedString();
            } else if (c == '{') {
                return scanTokenString();
            }
            myDriver.ungetChar();
            return scanWord();

        case '.':
            myDriver.nextChar();
            c = myDriver.getChar();
            if (c == '.') {
                myDriver.nextChar();
                c = myDriver.getChar();
                if (c == '.') {
                    myDriver.nextChar();
                    return TokenType.DOTDOTDOT;
                }
                return TokenType.DOTDOT;

            } else if (Character.isDigit(c)) {
                myDriver.ungetChar();
                return scanNumberLiteral();

            } else {
                return TokenType.DOT;
            }

        case '/':
            myDriver.nextChar();
            c = myDriver.getChar();
            switch (c) {
            case '*':
                myDriver.nextChar();
                return scanMultiLineComment();

            case '+':
                myDriver.nextChar();
                return scanNestedComment();

            case '/':
                myDriver.nextChar();
                return scanLineComment();

            case '=':
                myDriver.nextChar();
                return TokenType.DIVASS;

            default:
                return TokenType.DIV;
            }

        case '&':
            myDriver.nextChar();
            c = myDriver.getChar();
            switch (c) {
            case '&':
                myDriver.nextChar();
                return TokenType.ANDAND;

            case '=':
                myDriver.nextChar();
                return TokenType.ANDASS;

            default:
                return TokenType.AND;
            }

        case '|':
            myDriver.nextChar();
            c = myDriver.getChar();
            switch (c) {
            case '|':
                myDriver.nextChar();
                return TokenType.OROR;

            case '=':
                myDriver.nextChar();
                return TokenType.ORASS;

            default:
                return TokenType.OR;
            }

        case '-':
            myDriver.nextChar();
            c = myDriver.getChar();
            switch (c) {
            case '-':
                myDriver.nextChar();
                return TokenType.DECREMENT;

            case '=':
                myDriver.nextChar();
                return TokenType.MINUSASS;

            default:
                return TokenType.MINUS;
            }

        case '+':
            myDriver.nextChar();
            c = myDriver.getChar();
            switch (c) {
            case '+':
                myDriver.nextChar();
                return TokenType.INCREMENT;

            case '=':
                myDriver.nextChar();
                return TokenType.PLUSASS;

            default:
                return TokenType.PLUS;
            }

        case '<':
            myDriver.nextChar();
            c = myDriver.getChar();
            switch (c) {
            case '<':
                myDriver.nextChar();
                c = myDriver.getChar();
                if (c == '=') {
                    myDriver.nextChar();
                    return TokenType.SHLASS;
                }
                return TokenType.SHL;

            case '>':
                myDriver.nextChar();
                c = myDriver.getChar();
                if (c == '=') {
                    myDriver.nextChar();
                    return TokenType.LESSGREATEQ;
                }
                return TokenType.LESSGREAT;

            case '=':
                myDriver.nextChar();
                return TokenType.LESSEQ;

            default:
                return TokenType.LESS;
            }

        case '>':
            myDriver.nextChar();
            c = myDriver.getChar();
            switch (c) {
            case '>':
                myDriver.nextChar();
                c = myDriver.getChar();
                switch (c) {
                case '>':
                    myDriver.nextChar();
                    c = myDriver.getChar();
                    if (c == '=') {
                        myDriver.nextChar();
                        return TokenType.SHRUNSASS;
                    }
                    return TokenType.SHRUNS;

                case '=':
                    myDriver.nextChar();
                    return TokenType.SHRASS;

                default:
                    return TokenType.SHR;
                }

            case '=':
                myDriver.nextChar();
                return TokenType.GREATEQ;

            default:
                return TokenType.GREAT;
            }

        case '!':
            myDriver.nextChar();
            c = myDriver.getChar();
            switch (c) {
            case '=':
                myDriver.nextChar();
                return TokenType.NOTEQ;

            case '<':
                myDriver.nextChar();
                c = myDriver.getChar();
                switch (c) {
                case '>':
                    myDriver.nextChar();
                    c = myDriver.getChar();
                    if (c == '=') {
                        myDriver.nextChar();
                        return TokenType.NOTLESSGREATEQ;
                    }
                    return TokenType.NOTLESSGREAT;

                case '=':
                    myDriver.nextChar();
                    return TokenType.NOTLESSEQ;

                default:
                    return TokenType.NOTLESS;
                }

            case '>':
                myDriver.nextChar();
                c = myDriver.getChar();
                if (c == '=') {
                    myDriver.nextChar();
                    return TokenType.NOTGREATEQ;
                }
                return TokenType.NOTGREAT;

            case 'i':
                myDriver.nextChar();
                c = myDriver.getChar();
                if (c == 's') {
                    myDriver.nextChar();
                    c = myDriver.getChar();
                    if (!Character.isAlNum(c)) {
                        return TokenType.NOTIS;
                    }
                    myDriver.ungetChar();
                }
                myDriver.ungetChar();
                // fall through

            default:
                return TokenType.NOT;
            }

        case '(':
            myDriver.nextChar();
            return TokenType.OPEN_PAREN;

        case ')':
            myDriver.nextChar();
            return TokenType.CLOSE_PAREN;

        case '[':
            myDriver.nextChar();
            return TokenType.OPEN_BRACKET;

        case ']':
            myDriver.nextChar();
            return TokenType.CLOSE_BRACKET;

        case '{':
            myDriver.nextChar();
            return TokenType.OPEN_CURLY;

        case '}':
            myDriver.nextChar();
            return TokenType.CLOSE_CURLY;

        case '?':
            myDriver.nextChar();
            return TokenType.QUESTION;

        case ',':
            myDriver.nextChar();
            return TokenType.COMMA;

        case ';':
            myDriver.nextChar();
            return TokenType.SEMICOLON;

        case ':':
            myDriver.nextChar();
            return TokenType.COLON;

        case '$':
            myDriver.nextChar();
            return TokenType.DOLLAR;

        case '=':
            myDriver.nextChar();
            c = myDriver.getChar();
            if (c == '=') {
                myDriver.nextChar();
                return TokenType.EQ;
            }
            return TokenType.ASSIGN;

        case '*':
            myDriver.nextChar();
            c = myDriver.getChar();
            if (c == '=') {
                myDriver.nextChar();
                return TokenType.MULTASS;
            }
            return TokenType.STAR;

        case '%':
            myDriver.nextChar();
            c = myDriver.getChar();
            if (c == '=') {
                myDriver.nextChar();
                return TokenType.MODASS;
            }
            return TokenType.MOD;

        case '^':
            myDriver.nextChar();
            c = myDriver.getChar();
            if (c == '=') {
                myDriver.nextChar();
                return TokenType.CARETASS;
            } else if (c == '^') {
                myDriver.nextChar();
                if (myDriver.getChar() == '=') {
                    myDriver.nextChar();
                    return TokenType.CARETCARETASS;
                }
                return TokenType.CARETCARET;
            }
            return TokenType.CARET;

        case '~':
            myDriver.nextChar();
            c = myDriver.getChar();
            if (c == '=') {
                myDriver.nextChar();
                return TokenType.TILDEASS;
            }
            return TokenType.TILDE;

        case '@':
            myDriver.nextChar();
            string name = scanIdentifier();
            switch (name) {
            case "property":
                myDriver.nextChar();
                return TokenType.PROPERTY;
            default:
                return error("Unknown attribute");
            }

        case '#':
            myDriver.nextChar();
            if (myLineNumber == 0 && myDriver.getChar() == '!') {
                while (myDriver.getChar() != '\n') {
                    myDriver.nextChar();
                }
                myDriver.nextChar();
                return scan();
            }

            string name = scanIdentifier();
            if (name == "line") {
                //Integer + FileSpec
            }
            return error("Unknown special token sequence");

        case '\0':
            return TokenType.EOF;

        default:
            if (Character.isAlpha(c) || c == '_') {
                return scanWord();
            }

            if (Character.isDigit(c)) {
                return scanNumberLiteral();
            }

            return error("Invalid character");
        }
    }

    private TokenType scanWhitespaces() {
        char c = myDriver.getChar();
        string value = "";
        if (Character.isSpace(c)) {
            if (c == '\n') {
                ++myLineNumber;
            }
            value ~= c;
            myDriver.nextChar();
            c = myDriver.getChar();
        }
        myTokenValue = value;
        return TokenType.WHITESPACE;
    }

    private TokenType error(string message)
    in {
        assert(message != "");
    }
    body {
        char c = myDriver.getChar();
        while (Character.isAlNum(c) || c == '_' || c == '.') {
            myDriver.nextChar();
            c = myDriver.getChar();
        }
        myTokenValue = message;
        return TokenType.INVALID;
    }

    private TokenType scanWord() {
        string name = scanIdentifier();
        if (Patterns.isKeyword(name)) {
            return Patterns.getKeywordToken(name);
        } else {
            myTokenValue = name;
            return TokenType.IDENTIFIER;
        }
    }

    private string scanIdentifier()
    in {
        char c = myDriver.getChar();
        assert(Character.isAlpha(c) || c == '_');
    }
    body {
        string name;
        name ~= myDriver.getChar();
        myDriver.nextChar();
        char c = myDriver.getChar();
        while (Character.isAlNum(c) || c == '_') {
            name ~= c;
            myDriver.nextChar();
            c = myDriver.getChar();
        }
        return name;
    }

    // TODO: Optimize it
    private TokenType scanNumberLiteral()
    in {
        char c = myDriver.getChar();
        assert(Character.isDigit(c) || c == '.');
    }
    body {
        ubyte base = 10;
        string value;
        bool pointFlag = false;
        char c = myDriver.getChar();

        if (c == '0') {
            myDriver.nextChar();
            c = myDriver.getChar();
            if ((c == 'x') || (c == 'X')) {
                base = 16;
                myDriver.nextChar();
                c = myDriver.getChar();
            } else if ((c == 'b') || (c == 'B')) {
                base = 2;
                myDriver.nextChar();
                c = myDriver.getChar();
            } else {
                value ~= '0';
                base = 8;
            }
        }

        while ((Character.isAlNum(c)
                || (c == '_') || (c == '.'))
                && (c != 'p') && (c != 'P')
                && (c != 'u') && (c != 'U')
                && (c != 'l') && (c != 'L')
                && (c != 'i') && (c != 'I')) {
            if (c == '_') {
                myDriver.nextChar();
                c = myDriver.getChar();
                continue;
            }
            if (c == '.') {
                myDriver.nextChar();
                c = myDriver.getChar();
                if (c == '.') {
                    myDriver.ungetChar();
                    break;
                }

                if (pointFlag) {
                    return error("There are too many \'.\'");
                }
                pointFlag = true;
                if (base == 8) {
                    base = 10;
                } else if (base == 2) {
                    return error("Floating literal may not be in radix 2");
                }
            }
            if ((base != 16) && (c == 'e' || c == 'E' || c == 'f' || c == 'F')) {
                break;
            }
            value ~= c;
            myDriver.nextChar();
            c = myDriver.getChar();
        }

        string scanSignInt() {
            char c = myDriver.getChar();
            string res;
            if ((c == '-') || (c == '+')) {
                res ~= c;
                myDriver.nextChar();
                c = myDriver.getChar();
            }
            while (Character.isDigit(c) || c == '_') {
                if (c != '_') {
                    res ~= c;
                }
                myDriver.nextChar();
                c = myDriver.getChar();
            }
            return res;
        }

        if ((c == 'e') || (c == 'E')
                || (c == 'p') || (c == 'P')) {
            pointFlag = true;
            value ~= c;
            myDriver.nextChar();
            value ~= scanSignInt();
            c = myDriver.getChar();
        }

        try {
            if (pointFlag) {
                switch (c) {
                case 'f':
                case 'F':
                    myDriver.nextChar();
                    c = myDriver.getChar();
                    if ((c == 'i') || (c == 'I')) {
                        myDriver.nextChar();
                        myTokenValue = parseNumber!ifloat(value, base);
                        return TokenType.IFLOAT_LITERAL;
                    }
                    myTokenValue = parseNumber!float(value, base);
                    return TokenType.FLOAT_LITERAL;

                case 'l':
                    return error("Suffix \'l\' is deprecated, use \'L\' instead");
                case 'L':
                    myDriver.nextChar();
                    c = myDriver.getChar();
                    if ((c == 'i') || (c == 'I')) {
                        myDriver.nextChar();
                        myTokenValue = parseNumber!ireal(value, base);
                        return TokenType.IREAL_LITERAL;
                    }
                    myTokenValue = parseNumber!real(value, base);
                    return TokenType.REAL_LITERAL;

                case 'I':
                    return error("Suffix \'I\' is deprecated, use \'i\' instead");
                case 'i':
                    myDriver.nextChar();
                    myTokenValue = parseNumber!idouble(value, base);
                    return TokenType.IDOUBLE_LITERAL;

                default:
                    if (Character.isAlpha(c)) {
                        return error(format("Suffix %s may not using with floating literal", c));
                    }
                    myTokenValue = parseNumber!double(value, base);
                    return TokenType.DOUBLE_LITERAL;
                }
            } else {
                switch (c) {
                case 'u':
                case 'U':
                    myDriver.nextChar();
                    c = myDriver.getChar();
                    if ((c == 'l') || (c == 'L')) {
                        myDriver.nextChar();
                        myTokenValue = parseNumber!ulong(value, base);
                        return TokenType.ULONG_LITERAL;
                    }
                    myTokenValue = parseNumber!uint(value, base);
                    return TokenType.UINT_LITERAL;

                case 'l':
                case 'L':
                    myDriver.nextChar();
                    c = myDriver.getChar();
                    if ((c == 'u') || (c == 'U')) {
                        myDriver.nextChar();
                        myTokenValue = parseNumber!ulong(value, base);
                        return TokenType.ULONG_LITERAL;
                    }
                    myTokenValue = parseNumber!long(value, base);
                    return TokenType.LONG_LITERAL;

                default:
                    if (Character.isAlpha(c)) {
                        return error(format("Suffix %s may not using with integer literal", c));
                    }
                    myTokenValue = parseNumber!int(value, base);
                    return TokenType.INT_LITERAL;
                }
            }
        } catch (LexerException e) {
            return error(e.msg);
        }
    }

    // TODO: Correctly work with encoding
    private TokenType scanSingleCharacter()
    in {
        assert(myDriver.getChar() == '\'');
    }
    body {
        myDriver.nextChar();
        char c = myDriver.getChar();
        switch (c) {
        case '\n':
        case '\r':
        case 0:
        case '\'':
            return error("Unterminated character constant");

        case '\\':
            try {
                myTokenValue = scanEscapeSequence();
            } catch (LexerException e) {
                return error(e.msg);
            }
            break;

        default:
            myTokenValue = c;
        }
        myDriver.nextChar();
        c = myDriver.getChar();
        if (c != '\'') {
            return error("There is no \'");
        }
        myDriver.nextChar();
        return TokenType.CHARACTER_LITERAL;
    }

    // TODO: Support \EndOfFile
    // TODO: Check isValidChar
    private char scanEscapeSequence()
    in {
        assert(myDriver.getChar() == '\\');
    }
    body {
        myDriver.nextChar();
        char c = myDriver.getChar();
        switch (c) {
        case '\'':
            return '\'';
        case '\"':
            return '\"';
        case '\?':
            return '\?';
        case '\\':
            return '\\';
        case 'a':
            return '\a';
        case 'b':
            return '\b';
        case 'f':
            return '\f';
        case 'n':
            return '\n';
        case 'r':
            return '\r';
        case 't':
            return '\t';
        case 'v':
            return '\v';
        case 'x':
            return scanHexDigits(2u);
        case 'u':
            return scanHexDigits(4u);
        case 'U':
            return scanHexDigits(8u);
        case '&':
            return scanNamedEntity();
        default:
            if (Character.isDigit(c, 8)) {
                uint res;
                ubyte n = 0;
                do {
                    res <<= 3; // res *= 8;
                    myDriver.nextChar();
                    c = myDriver.getChar();
                } while (Character.isDigit(c, 8) && ++n < 3);
                return cast(char)res;
            }
            throw new LexerException("Undefined escape sequence");
        }
    }

    private char scanHexDigits(in ubyte ndigit)
    in {
        assert((ndigit & 1) == 0);
    }
    body {
        uint res;
        for (ubyte i = 0; i < ndigit; ++i) {
            myDriver.nextChar();
            char c = myDriver.getChar();
            if (!Character.isDigit(c, 16)) {
                throw new LexerException("Invalid hex digit");
            }
            res <<= 4;  // res *= 16;
        }
        if (!isValidDchar(res)) {
            throw new LexerException("Invalid UTF character");
        }
        return cast(char)res;
    }

    private char scanNamedEntity()
    in {
        assert(myDriver.getChar() == '&');
    }
    body {
        myDriver.nextChar();
        char c = myDriver.getChar();
        string res;
        while (c != ';') {
            res ~= c;
            myDriver.nextChar();
            c = myDriver.getChar();
        }
        myDriver.nextChar();
        if (!Patterns.isNamedEntity(res)) {
            throw new LexerException("There are no such named entity");
        }
        return Patterns.getNamedEntitySymbol(res);
    }

    private TokenType scanDoubleQuotedString()
    in {
        assert(myDriver.getChar() == '\"');
    }
    body {
        myDriver.nextChar();
        char c = myDriver.getChar();
        string tokenValue;
        while (c != '\"') {
            if (c == 0) {
                return error("Unterminated string constant");
            }
            if (c == '\\') {
                c = scanEscapeSequence();
            }
            myDriver.nextChar();
            tokenValue ~= c;
            c = myDriver.getChar();
        }
        myTokenValue = tokenValue;
        myDriver.nextChar();
        return scanSuffix();
    }

    private TokenType scanWysiwygString()
    in {
        assert(myDriver.getChar() == '\"' || myDriver.getChar() == '`');
    }
    body {
        char terminalChar = myDriver.getChar();
        myDriver.nextChar();
        char c = myDriver.getChar();
        string tokenValue;
        while (c != terminalChar) {
            if (c == 0) {
                return error("Unterminated string constant");
            }
            myDriver.nextChar();
            tokenValue ~= c;
            c = myDriver.getChar();
        }
        myTokenValue = tokenValue;
        myDriver.nextChar();
        return scanSuffix();
    }

    // TODO: Correct value
    private TokenType scanHexString()
    in {
        assert(myDriver.getChar() == '\"');
    }
    body {
        myDriver.nextChar();
        char c = myDriver.getChar();
        string tokenValue;
        while (c != '\"') {
            if (c == 0) {
                return error("Unterminated string constant");
            }
            if (Character.isGraph(c)) {
                tokenValue ~= c;
            }
            myDriver.nextChar();
            c = myDriver.getChar();
        }
        myTokenValue = tokenValue;
        myDriver.nextChar();
        return scanSuffix();
    }

    private TokenType scanSuffix() {
        char c = myDriver.getChar();
        TokenType res;
        switch (c) {
        case 'c':
            res = TokenType.STRING_LITERAL;
            break;
        case 'w':
            res = TokenType.WSTRING_LITERAL;
            break;
        case 'd':
            res = TokenType.DSTRING_LITERAL;
            break;
        default:
            return TokenType.STRING_LITERAL;
        }
        myDriver.nextChar();
        return res;
    }

    // TODO: Identifiers delemiters support
    // TODO: Add token value
    // TODO: Bug in docs or dmd, such strings cannot have suffix
    private TokenType scanDelemitedString()
    in {
        assert(myDriver.getChar() == '\"');
    }
    body {
        myDriver.nextChar();
        char c = myDriver.getChar();
        char delemiterLeft;
        char delemiterRight;
        ushort nestCount;
        bool nestingFlag = true;
        delemiterLeft = c;

        switch (c) {
        case '<':
            delemiterRight = '>';
            break;
        case '[':
            delemiterRight = ']';
            break;
        case '(':
            delemiterRight = ')';
            break;
        case '{':
            delemiterRight = '}';
            break;
        default:
            if (Character.isAlpha(c) || c == '_') {
                scanIdentifier();
            } else if (Character.isSpace(c)) {
                return error("Delemiter cannot be whitespace");
            }
            delemiterRight = delemiterLeft;
            nestingFlag = false;
        }

        myDriver.nextChar();
        c = myDriver.getChar();
        while ((c != delemiterRight) || (nestCount != 0)) {
            if (c == 0) {
                return error("Unterminated string constant");
            }
            if (nestingFlag) {
                if (c == delemiterLeft) {
                    ++nestCount;
                } else if (c == delemiterRight) {
                    --nestCount;
                }
            }
            myDriver.nextChar();
            c = myDriver.getChar();
        }

        myDriver.nextChar();
        c = myDriver.getChar();
        if (c != '\"') {
            return error("Unterminated string constant");
        }
        myDriver.nextChar();
        return scanSuffix();
    }

    // TODO: Correct support
    private TokenType scanTokenString()
    in {
        assert(myDriver.getChar() == '{');
    }
    body {
        myDriver.nextChar();
        char c = myDriver.getChar();
        while (c != '}') {
            if (c == 0) {
                return error("Unterminated string constant");
            }
            myDriver.nextChar();
            c = myDriver.getChar();
        }
        myDriver.nextChar();
        return TokenType.STRING_LITERAL;
    }

    private TokenType scanLineComment() {
        bool ddocFlag = false;
        char c = myDriver.getChar();
        if (c == '/') {
            ddocFlag = true;
            myDriver.nextChar();
            c = myDriver.getChar();
        }
        while ((c != '\n') && (c != 0)) {
            myDriver.nextChar();
            c = myDriver.getChar();
        }
        return (ddocFlag) ? TokenType.DOC_COMMENT : TokenType.COMMENT;
    }

    // TODO: Remove stars from ddoc comment
    private TokenType scanMultiLineComment() {
        bool ddocFlag = false;
        char c = myDriver.getChar();
        if (c == '*') {
            ddocFlag = true;
            myDriver.nextChar();
            c = myDriver.getChar();
        }
        while (c != 0) {
            if (c == '*') {
                myDriver.nextChar();
                c = myDriver.getChar();
                if (c == '/') {
                    myDriver.nextChar();
                    break;
                }
            }
            myDriver.nextChar();
            c = myDriver.getChar();
        }
        return (ddocFlag) ? TokenType.DOC_COMMENT : TokenType.COMMENT;
    }

    // TODO: Remove pluses from ddoc comment
    private TokenType scanNestedComment() {
        bool ddocFlag = false;
        char c = myDriver.getChar();
        if (c == '+') {
            ddocFlag = true;
            myDriver.nextChar();
            c = myDriver.getChar();
        }
        ubyte nestCount = 1;
        while (c != 0) {
            if (c == '+') {
                myDriver.nextChar();
                c = myDriver.getChar();
                if (c == '/') {
                    myDriver.nextChar();
                    if (--nestCount == 0) {
                        break;
                    }
                }
            }
            myDriver.nextChar();
            c = myDriver.getChar();
        }
        return (ddocFlag) ? TokenType.DOC_COMMENT : TokenType.COMMENT;
    }

    unittest {
        DLexer lexer = new DLexer();
        lexer.analize("\"Hello\"");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.STRING_LITERAL);
        lexer.analize("r\"Hello\"");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.STRING_LITERAL);
        lexer.analize("x\"x89 x54 x43\"");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.STRING_LITERAL);
        lexer.analize("q\"{Hello{}}\"");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.STRING_LITERAL);

        lexer.analize("\"Hello\"d");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.DSTRING_LITERAL);
        lexer.analize("r\"Hello\"d");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.DSTRING_LITERAL);
        lexer.analize("x\"x89 x54 x43\"d");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.DSTRING_LITERAL);
        lexer.analize("q\"{Hello{}}\"d");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.DSTRING_LITERAL);

        lexer.analize("\"Hello\"w");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.WSTRING_LITERAL);
        lexer.analize("r\"Hello\"w");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.WSTRING_LITERAL);
        lexer.analize("x\"x89 x54 x43\"w");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.WSTRING_LITERAL);
        lexer.analize("q\"{Hello{}}\"w");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.WSTRING_LITERAL);

        lexer.analize("hello");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.IDENTIFIER);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.EOF);
        lexer.analize("_h___e34llo678689");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.IDENTIFIER);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.EOF);

        lexer.analize("12a");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.INVALID);
        lexer.analize("0.12p1");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.INVALID);

        lexer.analize("// some comment \n notComment");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.COMMENT);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.IDENTIFIER);
        lexer.analize("/* some comment \n comment */ notComment");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.COMMENT);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.IDENTIFIER);
        lexer.analize("/+ some comment \n /+ comment +/ comment \n +/ notComment");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.COMMENT);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.IDENTIFIER);

        lexer.analize("/// some comment \n notComment");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.DOC_COMMENT);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.IDENTIFIER);
        lexer.analize("/** some comment \n comment */ notComment");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.DOC_COMMENT);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.IDENTIFIER);
        lexer.analize("/++ some comment \n /+ comment +/ comment \n +/ notComment");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.DOC_COMMENT);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.IDENTIFIER);

        lexer.analize(r"'a' '\a' '\x86''\u8786' '\U65875743' '\U6587574'");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.CHARACTER_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.CHARACTER_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.CHARACTER_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.CHARACTER_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.CHARACTER_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.INVALID);

        lexer.analize(r"123_456_789 0xABC_DEF 04357 0b1001110_1");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.INT_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.INT_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.INT_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.INT_LITERAL);

        lexer.analize(r"123_456_789u 0xABC_DEFu 04357u 0b1001110_1u");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.UINT_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.UINT_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.UINT_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.UINT_LITERAL);

        lexer.analize(r"123_456_789l 0xABC_DEFl 04357L 0b1001110_1l");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.LONG_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.LONG_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.LONG_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.LONG_LITERAL);

        lexer.analize(r"123_456_789uL 0xABC_DEFUl 04357ul 0b1001110_1UL");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.ULONG_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.ULONG_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.ULONG_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.ULONG_LITERAL);

        lexer.analize(r"34.5 0xaP9 56.7e2_7 67.e+9 986_076.12e-1_7 0b0.5");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.DOUBLE_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.DOUBLE_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.DOUBLE_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.DOUBLE_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.DOUBLE_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.INVALID);

        lexer.analize(r"34.5f 0xaP9f 56.7e2_7f 67.e+9F 986_076.12e-1_7f .12_599L 0b0.5f");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.FLOAT_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.FLOAT_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.FLOAT_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.FLOAT_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.FLOAT_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.REAL_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.INVALID);

        lexer.analize(r"34.5i 0xaP9i 56.7e2_7i 67.e+9i 986_076.12e-1_7i 0b0.5i");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.IDOUBLE_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.IDOUBLE_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.IDOUBLE_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.IDOUBLE_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.IDOUBLE_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.INVALID);

        lexer.analize(r"!is !isnt");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.NOTIS);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.NOT);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.IDENTIFIER);

        lexer.analize(r"! == = != !<>= !<> !> !< && || .. ... . ^ += ^= &= |= * + / /= % $ ~ , ; : [?] ^^ ^^=");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.NOT);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.EQ);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.ASSIGN);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.NOTEQ);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.NOTLESSGREATEQ);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.NOTLESSGREAT);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.NOTGREAT);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.NOTLESS);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.ANDAND);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.OROR);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.DOTDOT);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.DOTDOTDOT);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.DOT);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.CARET);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.PLUSASS);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.CARETASS);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.ANDASS);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.ORASS);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.STAR);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.PLUS);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.DIV);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.DIVASS);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.MOD);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.DOLLAR);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.TILDE);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.COMMA);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.SEMICOLON);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.COLON);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.OPEN_BRACKET);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.QUESTION);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.CLOSE_BRACKET);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.CARETCARET);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.CARETCARETASS);

        lexer.analize(" @property ");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.PROPERTY);
    }

    unittest {
        DLexer lexer = new DLexer();
        lexer.analize("12a a 0b2 a 0b0.1 a");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.INVALID);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.IDENTIFIER);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.INVALID);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.IDENTIFIER);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.INVALID);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.IDENTIFIER);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.EOF);
    }

    unittest {
        DLexer lexer = new DLexer();
        lexer.analize("import std.stdio; \n int main() {\nwriteln(\"Hello world\"); \nreturn 0;\n}");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.IMPORT);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.IDENTIFIER);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.DOT);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.IDENTIFIER);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.SEMICOLON);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.INT);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.IDENTIFIER);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.OPEN_PAREN);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.CLOSE_PAREN);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.OPEN_CURLY);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.IDENTIFIER);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.OPEN_PAREN);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.STRING_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.CLOSE_PAREN);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.SEMICOLON);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.RETURN);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.INT_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.SEMICOLON);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.CLOSE_CURLY);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.EOF);
    }

    unittest {
        assert(parseNumber!float("0.1") == 0.1f);
        assert(parseNumber!ifloat("0.1") == 0.1fi);
        assert(parseNumber!idouble("0.1") == 0.1i);
        assert(parseNumber!ireal("0.1") == 0.1Li);
        assert(parseNumber!int("100866") == 100_866);
        assert(parseNumber!float("0.1E2") == 0.1e2f);
        assert(parseNumber!float("0.1E+2") == 0.1e+2f);
        assert(parseNumber!float("0.1e-2") == 0.1e-2f);
        assert(parseNumber!float("3.1") == 3.1f);
        assert(parseNumber!float(".1") == .1f);
        assert(parseNumber!real("1234567890.1234567890123456789") == 1234567890.1234567890123456789L);
        assert(parseNumber!double("56.7e97") == 56.7e97);
        assert(parseNumber!double("56.7p1010", 16) == 0x56.7p1010);
        assert(parseNumber!int("abcdef", 16) == 0xabcdef);
        assert(parseNumber!double("5aC6.7p10", 16) == 0x5aC6.7p10);
    }

    unittest {
        DLexer lexer = new DLexer();
        lexer.analize("a[1121..3124]");
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.IDENTIFIER);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.OPEN_BRACKET);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.INT_LITERAL);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.DOTDOT);
        lexer.nextToken();
        assert(lexer.getTokenType() == TokenType.INT_LITERAL);
    }
}
